Node / ReticulumProjects / MeshChatX.git / files / tests / backend / test_memory_profiling.py

Displaying Raw • Download

T8b949e# SPDX-License-Identifier: 0BSD

Tff7b72import T7ee787os

Tff7b72import T7ee787secrets

Tff7b72import T7ee787shutil

Tff7b72import T7ee787tempfile

Tff7b72import T7ee787unittest

Tff7b72from T7ee787meshchatxT7ee787.T7ee787srcT7ee787.T7ee787backendT7ee787.T7ee787database Tff7b72import Te6edf3Database

Tff7b72from T7ee787meshchatxT7ee787.T7ee787srcT7ee787.T7ee787backendT7ee787.T7ee787identity_manager Tff7b72import Te6edf3IdentityManager

Tff7b72from T7ee787testsT7ee787.T7ee787backendT7ee787.T7ee787benchmarking_utils Tff7b72import Te6edf3MemoryTracker

Tff7b72class T56d364TestMemoryProfilingTb4b4b4(Te6edf3unittestTff7b72.Td2a8ffTestCaseTb4b4b4)Tb4b4b4:

Tff7b72def Td2a8ffsetUpTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:

Tff7b72selfTff7b72.Td2a8fftest_dir Tff7b72= Te6edf3tempfileTff7b72.Td2a8ffmkdtempTb4b4b4(Tb4b4b4)

Tff7b72selfTff7b72.Td2a8ffdb_path Tff7b72= Te6edf3osTff7b72.Td2a8ffpathTff7b72.Td2a8ffjoinTb4b4b4(Tff7b72selfTff7b72.Td2a8fftest_dirTb4b4b4, Ta5d6ff"Ta5d6fftest.dbTa5d6ff"Tb4b4b4)

Tff7b72selfTff7b72.Td2a8ffdb Tff7b72= Te6edf3DatabaseTb4b4b4(Tff7b72selfTff7b72.Td2a8ffdb_pathTb4b4b4)

Tff7b72selfTff7b72.Td2a8ffdbTff7b72.Td2a8ffinitializeTb4b4b4(Tb4b4b4)

Tff7b72def Td2a8fftearDownTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:

Tff7b72selfTff7b72.Td2a8ffdbTff7b72.Td2a8ffclose_allTb4b4b4(Tb4b4b4)

Tff7b72if Te6edf3osTff7b72.Td2a8ffpathTff7b72.Td2a8ffexistsTb4b4b4(Tff7b72selfTff7b72.Td2a8fftest_dirTb4b4b4)Tb4b4b4:

Te6edf3shutilTff7b72.Td2a8ffrmtreeTb4b4b4(Tff7b72selfTff7b72.Td2a8fftest_dirTb4b4b4)

Tff7b72def Td2a8fftest_database_growthTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:

T8b949e"""Profile memory growth during heavy database insertions."""

Tff7b72with Te6edf3MemoryTrackerTb4b4b4(Ta5d6ff"Ta5d6ffDatabase Growth (10k messages)Ta5d6ff"Tb4b4b4) Tff7b72as Te6edf3trackerTb4b4b4:

Te6edf3num_messages Tff7b72= T79c0ff10000

Te6edf3peer_hash Tff7b72= Te6edf3secretsTff7b72.Td2a8fftoken_hexTb4b4b4(T79c0ff16Tb4b4b4)

T8b949e# Using transaction for speed, but tracking overall memory

Tff7b72with Tff7b72selfTff7b72.Td2a8ffdbTff7b72.Td2a8ffproviderTb4b4b4:

Tff7b72for Te6edf3i Tff7b72in Tffa657rangeTb4b4b4(Te6edf3num_messagesTb4b4b4)Tb4b4b4:

Te6edf3msg Tff7b72= Tb4b4b4{

Ta5d6ff"Ta5d6ffhashTa5d6ff"Tb4b4b4: Te6edf3secretsTff7b72.Td2a8fftoken_hexTb4b4b4(T79c0ff16Tb4b4b4)Tb4b4b4,

Ta5d6ff"Ta5d6ffsource_hashTa5d6ff"Tb4b4b4: Te6edf3peer_hashTb4b4b4,

Ta5d6ff"Ta5d6ffdestination_hashTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmy_hashTa5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6ffpeer_hashTa5d6ff"Tb4b4b4: Te6edf3peer_hashTb4b4b4,

Ta5d6ff"Ta5d6ffstateTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffdeliveredTa5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6ffprogressTa5d6ff"Tb4b4b4: T79c0ff1.0Tb4b4b4,

Ta5d6ff"Ta5d6ffis_incomingTa5d6ff"Tb4b4b4: T79c0ff1Tb4b4b4,

Ta5d6ff"Ta5d6ffmethodTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffdirectTa5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6ffdelivery_attemptsTa5d6ff"Tb4b4b4: T79c0ff1Tb4b4b4,

Ta5d6ff"Ta5d6fftitleTa5d6ff"Tb4b4b4: Ta5d6fffTa5d6ff"Ta5d6ffMsg Tffd700{Te6edf3iTffd700}Ta5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6ffcontentTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffATa5d6ff" Tff7b72* T79c0ff512Tb4b4b4, T8b949e# 512 bytes content

Ta5d6ff"Ta5d6fffieldsTa5d6ff"Tb4b4b4: Ta5d6ff"Tffd700{}Ta5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6fftimestampTa5d6ff"Tb4b4b4: T79c0ff1234567890 Tff7b72+ Te6edf3iTb4b4b4,

Ta5d6ff"Ta5d6ffrssiTa5d6ff"Tb4b4b4: Tff7b72-T79c0ff50Tb4b4b4,

Ta5d6ff"Ta5d6ffsnrTa5d6ff"Tb4b4b4: T79c0ff5.0Tb4b4b4,

Ta5d6ff"Ta5d6ffqualityTa5d6ff"Tb4b4b4: T79c0ff3Tb4b4b4,

Ta5d6ff"Ta5d6ffis_spamTa5d6ff"Tb4b4b4: T79c0ff0Tb4b4b4,

Tb4b4b4}

Tff7b72selfTff7b72.Td2a8ffdbTff7b72.Td2a8ffmessagesTff7b72.Td2a8ffupsert_lxmf_messageTb4b4b4(Te6edf3msgTb4b4b4)

T8b949e# We expect some growth due to DB internal caching, but not excessive

T8b949e# 10k messages * 512 bytes is ~5MB of raw content.

T8b949e# SQLite should handle this efficiently.

Tff7b72selfTff7b72.Td2a8ffassertLessTb4b4b4(

Te6edf3trackerTff7b72.Td2a8ffmem_deltaTb4b4b4,

T79c0ff20.0Tb4b4b4,

Ta5d6ff"Ta5d6ffExcessive memory growth during DB insertionTa5d6ff"Tb4b4b4,

Tb4b4b4)

Tff7b72def Td2a8fftest_identity_manager_memoryTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:

T8b949e"""Profile memory usage of identity manager with many identities."""

Te6edf3manager Tff7b72= Te6edf3IdentityManagerTb4b4b4(Tff7b72selfTff7b72.Td2a8fftest_dirTb4b4b4)

Tff7b72with Te6edf3MemoryTrackerTb4b4b4(Ta5d6ff"Ta5d6ffIdentity Manager (50 identities)Ta5d6ff"Tb4b4b4) Tff7b72as Te6edf3trackerTb4b4b4:

Tff7b72for Te6edf3i Tff7b72in Tffa657rangeTb4b4b4(T79c0ff50Tb4b4b4)Tb4b4b4:

Te6edf3managerTff7b72.Td2a8ffcreate_identityTb4b4b4(Ta5d6fffTa5d6ff"Ta5d6ffProfile Tffd700{Te6edf3iTffd700}Ta5d6ff"Tb4b4b4)

T8b949e# Listing all identities

Te6edf3identities Tff7b72= Te6edf3managerTff7b72.Td2a8fflist_identitiesTb4b4b4(Tb4b4b4)

Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Tffa657lenTb4b4b4(Te6edf3identitiesTb4b4b4)Tb4b4b4, T79c0ff50Tb4b4b4)

Tff7b72selfTff7b72.Td2a8ffassertLessTb4b4b4(

Te6edf3trackerTff7b72.Td2a8ffmem_deltaTb4b4b4,

T79c0ff10.0Tb4b4b4,

Ta5d6ff"Ta5d6ffIdentity management consumed too much memoryTa5d6ff"Tb4b4b4,

Tb4b4b4)

Tff7b72def Td2a8fftest_large_message_processingTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:

T8b949e"""Profile memory when handling very large messages."""

Te6edf3large_content Tff7b72= Ta5d6ff"Ta5d6ffBTa5d6ff" Tff7b72* Tb4b4b4(T79c0ff1024 Tff7b72* T79c0ff1024Tb4b4b4) T8b949e# 1MB message

Te6edf3peer_hash Tff7b72= Te6edf3secretsTff7b72.Td2a8fftoken_hexTb4b4b4(T79c0ff16Tb4b4b4)

Tff7b72with Te6edf3MemoryTrackerTb4b4b4(Ta5d6ff"Ta5d6ffLarge Message (1MB)Ta5d6ff"Tb4b4b4) Tff7b72as Te6edf3trackerTb4b4b4:

Te6edf3msg Tff7b72= Tb4b4b4{

Ta5d6ff"Ta5d6ffhashTa5d6ff"Tb4b4b4: Te6edf3secretsTff7b72.Td2a8fftoken_hexTb4b4b4(T79c0ff16Tb4b4b4)Tb4b4b4,

Ta5d6ff"Ta5d6ffsource_hashTa5d6ff"Tb4b4b4: Te6edf3peer_hashTb4b4b4,

Ta5d6ff"Ta5d6ffdestination_hashTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmy_hashTa5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6ffpeer_hashTa5d6ff"Tb4b4b4: Te6edf3peer_hashTb4b4b4,

Ta5d6ff"Ta5d6ffstateTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffdeliveredTa5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6ffprogressTa5d6ff"Tb4b4b4: T79c0ff1.0Tb4b4b4,

Ta5d6ff"Ta5d6ffis_incomingTa5d6ff"Tb4b4b4: T79c0ff1Tb4b4b4,

Ta5d6ff"Ta5d6ffmethodTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffdirectTa5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6ffdelivery_attemptsTa5d6ff"Tb4b4b4: T79c0ff1Tb4b4b4,

Ta5d6ff"Ta5d6fftitleTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffLarge MessageTa5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6ffcontentTa5d6ff"Tb4b4b4: Te6edf3large_contentTb4b4b4,

Ta5d6ff"Ta5d6fffieldsTa5d6ff"Tb4b4b4: Ta5d6ff"Tffd700{}Ta5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6fftimestampTa5d6ff"Tb4b4b4: T79c0ff1234567890Tb4b4b4,

Ta5d6ff"Ta5d6ffrssiTa5d6ff"Tb4b4b4: Tff7b72-T79c0ff50Tb4b4b4,

Ta5d6ff"Ta5d6ffsnrTa5d6ff"Tb4b4b4: T79c0ff5.0Tb4b4b4,

Ta5d6ff"Ta5d6ffqualityTa5d6ff"Tb4b4b4: T79c0ff3Tb4b4b4,

Ta5d6ff"Ta5d6ffis_spamTa5d6ff"Tb4b4b4: T79c0ff0Tb4b4b4,

Tb4b4b4}

Tff7b72selfTff7b72.Td2a8ffdbTff7b72.Td2a8ffmessagesTff7b72.Td2a8ffupsert_lxmf_messageTb4b4b4(Te6edf3msgTb4b4b4)

T8b949e# Fetch it back

Te6edf3fetched Tff7b72= Tff7b72selfTff7b72.Td2a8ffdbTff7b72.Td2a8ffmessagesTff7b72.Td2a8ffget_lxmf_message_by_hashTb4b4b4(Te6edf3msgTb4b4b4[Ta5d6ff"Ta5d6ffhashTa5d6ff"Tb4b4b4]Tb4b4b4)

Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Tffa657lenTb4b4b4(Te6edf3fetchedTb4b4b4[Ta5d6ff"Ta5d6ffcontentTa5d6ff"Tb4b4b4]Tb4b4b4)Tb4b4b4, Tffa657lenTb4b4b4(Te6edf3large_contentTb4b4b4)Tb4b4b4)

T8b949e# 1MB message shouldn't cause much more than a few MBs of overhead

Tff7b72selfTff7b72.Td2a8ffassertLessTb4b4b4(Te6edf3trackerTff7b72.Td2a8ffmem_deltaTb4b4b4, T79c0ff5.0Tb4b4b4, Ta5d6ff"Ta5d6ffLarge message handling leaked memoryTa5d6ff"Tb4b4b4)

Tff7b72def Td2a8fftest_announce_manager_leaksTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:

T8b949e"""Test for memory leaks in AnnounceManager during repeated updates."""

Tff7b72with Te6edf3MemoryTrackerTb4b4b4(Ta5d6ff"Ta5d6ffAnnounce Stress (2k unique announces)Ta5d6ff"Tb4b4b4) Tff7b72as Te6edf3trackerTb4b4b4:

Tff7b72for Te6edf3i Tff7b72in Tffa657rangeTb4b4b4(T79c0ff2000Tb4b4b4)Tb4b4b4:

Te6edf3data Tff7b72= Tb4b4b4{

Ta5d6ff"Ta5d6ffdestination_hashTa5d6ff"Tb4b4b4: Te6edf3secretsTff7b72.Td2a8fftoken_hexTb4b4b4(T79c0ff16Tb4b4b4)Tb4b4b4,

Ta5d6ff"Ta5d6ffaspectTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6fflxmf.deliveryTa5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6ffidentity_hashTa5d6ff"Tb4b4b4: Te6edf3secretsTff7b72.Td2a8fftoken_hexTb4b4b4(T79c0ff16Tb4b4b4)Tb4b4b4,

Ta5d6ff"Ta5d6ffidentity_public_keyTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffpubkeyTa5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6ffapp_dataTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffsome data Ta5d6ff" Tff7b72* T79c0ff10Tb4b4b4,

Ta5d6ff"Ta5d6ffrssiTa5d6ff"Tb4b4b4: Tff7b72-T79c0ff50Tb4b4b4,

Ta5d6ff"Ta5d6ffsnrTa5d6ff"Tb4b4b4: T79c0ff5.0Tb4b4b4,

Ta5d6ff"Ta5d6ffqualityTa5d6ff"Tb4b4b4: T79c0ff3Tb4b4b4,

Tb4b4b4}

Tff7b72selfTff7b72.Td2a8ffdbTff7b72.Td2a8ffannouncesTff7b72.Td2a8ffupsert_announceTb4b4b4(Te6edf3dataTb4b4b4)

Tff7b72selfTff7b72.Td2a8ffassertLessTb4b4b4(

Te6edf3trackerTff7b72.Td2a8ffmem_deltaTb4b4b4,

T79c0ff15.0Tb4b4b4,

Ta5d6ff"Ta5d6ffAnnounce updates causing memory bloatTa5d6ff"Tb4b4b4,

Tb4b4b4)

Tff7b72if Tff7b72__name__ Tff7b72== Ta5d6ff"Ta5d6ff__main__Ta5d6ff"Tb4b4b4:

Te6edf3unittestTff7b72.Td2a8ffmainTb4b4b4(Tb4b4b4)